﻿

#r "FSharp.PowerPack.dll"
#r @"FSharp.PowerPack.Compatibility"
#r @"D:\WORK\MyFSharpExamples\Small\..\..\..\Business\references\books\F# and FP\FSharp-1.9.7.8\source\fsppack\FSharp.PowerPack\math\lapack\obj\Release\FSharp.PowerPack.Math.Providers.dll"
#r "System.Windows.Forms.DataVisualization.dll"
#r @"D:\WORK\libML\libML\CsLibrary\bin\Debug\CsLibrary.dll"
#r "FSharp.PowerPack.Parallel.Seq.dll"

//#r @"D:\WORK\libML\libML\Debug\libsvm.dll"
#r @"D:\WORK\libML\libsvm\Release\libsvm.dll"

#load "fatlab.fs"
#load "matrixExt.fs"
#load "libMLcommon.fs"
#load "utility.fs"
#load "io.fs"
#load "distribution.fs"
#load "dataset.fs"


open Microsoft.FSharp.Math
open Microsoft.FSharp.Math.Experimental
open libML
open libML.IO
open libML.Utility
open libML.Math

open System.IO
open System.Collections.Generic

let dim = 2000
let nAux = 19000
let auxImages = new Dictionary<int, svector>()

let auxImageLines = File.ReadAllLines @"D:\Research\aaai10\imageData\image_feature_mat.txt"
auxImageLines 
|> Seq.skip 1
|> Seq.map (fun line -> let a = line.Split ' ' in int(a.[0]), int(a.[1]), float(a.[2]))
|> Seq.groupBy (fun (i,j,v) -> i)
|> Seq.iter (fun ( i, s) ->
    let s2 = s |> Seq.map (fun (i,j,v) -> (j-1, v))
    let v = SVector.initFromPairSeq s2 2000
    auxImages.Add(i-1, v)
    )

let str2svector (s:string) = 
    SVector.initFromPairSeq (s.Split ' ' |> Seq.map (fun l -> let a = l.Split ':' in int(a.[0])-1, float(a.[1]))) dim

    
let topImages k query = 
    let dists = 
        Array.init nAux (fun i -> 
            match auxImages.TryGetValue(i) with
            | true, v -> SVector.cosineDist v query
            | false, _ -> failwith "impossible"
            )
    let sorted, idx = Array.sortReturnsIndex dists
    idx.[0..k-1]



let tagsLines = File.ReadAllLines @"D:\Research\aaai10\imageData\tag_image_mat.txt"
let tagImage = Array.init 20000 (fun _ -> new ResizeArray<int>())
tagsLines |> Seq.map (fun line -> let a = line.Split ' ' in int(a.[0])-1, int(a.[1])-1) |> Seq.iter (fun (tag,image) -> tagImage.[image].Add(tag))

let topTags k (images:int array) =
    let tags = seq {
        for i=0 to images.Length-1 do
            yield! ( tagImage.[images.[i]] |> ResizeArray.toSeq )
        }
    printfn "%A" tags
    let sorted = tags |> Seq.groupBy (fun i -> i) |> Seq.map (fun (i,v) -> i, Seq.length v) |> Seq.sortBy (fun (i,l) -> (-l, i)) |> Seq.toArray 
    sorted.[0..k-1]


let tagNames = File.ReadAllLines @"D:\Research\aaai10\imageData\stdWordsNew.txt"
let tag2name tagId = tagNames.[tagId]
    
let topTagNames query = 
    query |> str2svector |> topImages 100 |> topTags 30 |> Array.map (fun (tag,cnt) -> (tag2name tag, cnt))



// test

let cd1 = File.ReadAllText "cd1.txt"
let cd2 = File.ReadAllText "cd2.txt"
let cd3 = File.ReadAllText "cd3.txt"


let dog1 = File.ReadAllText "dog1.txt"
let dog2 = File.ReadAllText "dog2.txt"
let dog3 = File.ReadAllText "dog3.txt"


cd1 |> topTagNames
cd2 |> topTagNames
cd3 |> topTagNames


dog1 |> topTagNames
dog2 |> topTagNames
dog3 |> topTagNames



open System.IO

let imageFolder = @"D:\WORK\ImageData\flickr\"

let getFileNames folder = 
    let rec get folder = 
        seq {
            yield! (Directory.EnumerateFiles(folder, "*.jpg"))
            for sub in Directory.EnumerateDirectories folder do
                yield! (get sub)
        }
    get folder

 

File.WriteAllLines(@"D:\WORK\ImageData\flikr.list", getFileNames imageFolder)






let renameFilesInFolder (folder:string) = 
    let root = @"D:\WORK\ImageData\flickr\"
    let path = root + folder + "\\"
    let path2 = root + folder + "2\\"
    let files = Directory.EnumerateFiles(path, "*.jpg")
    files 
    |> Seq.iteri (fun i jpg ->
        let f1 = new FileInfo(jpg)
        if f1.Exists then 
            f1.MoveTo(path2 + i.ToString() + ".jpg")

            let tag = jpg + ".tag"
            let f2 = new FileInfo(tag)
            if f2.Exists then
                f2.MoveTo(path2 + i.ToString() + ".tag")
            else
                printfn "tag %d" i
        else
            printfn "jpg %d" i
    )

    


renameFilesInFolder "zebra"




#I @"D:\WORK\libML\libML\LuLibrary\bin\Release"
#r @"LuLibrary.dll"

let matr1 = HTL.DocTag.Main([| @"D:\WORK\ImageData\googleResults\"; "-d"; "-f"; "dog"; "-t"; "dog"; "animal"  |]);

let matr, nameList, tagList = HTL.ImageTag.Main([| @"D:\WORK\ImageData\flickr\"; "cd"; "dog" |])


let sifts = @"D:\WORK\htc\sampledSifts.txt"
let mat = @"D:\WORK\htc\sampledSiftsMat.txt"

let summary matrixName =    
    let lines = File.ReadLines(matrixName)
    lines 
    |> Seq.map (fun line -> 
        let s = line.Split([|' '|], System.StringSplitOptions.RemoveEmptyEntries)
        if s.Length % 2 <> 0 then failwith "must be even"
        s.Length/2) 
    |> Seq.sum

summary sifts

//732965 128  67145910

let toMatSparse matrixName (matName:string) = 
    let lines = File.ReadLines(matrixName)
    let out = new StreamWriter(matName)

    lines 
    |> Seq.skip 1
    |> Seq.iteri (fun i line -> 
        let s = line.Split([|' '|], System.StringSplitOptions.RemoveEmptyEntries)
        for j=0 to s.Length-1 do
            if j%2=0 then
                out.WriteLine((i+1).ToString() + " " + s.[j] + " " + s.[j+1])

        )
    out.Close()

toMatSparse sifts  mat



let lines = File.ReadAllLines @"D:\WORK\ImageData\categoryList.txt"
let lines2 = lines |> Seq.map (fun line -> line.Split '\t' ) |> Seq.toArray
let n = lines.Length
let taskFolder = @"D:\WORK\ImageData\task2\";
let mutable cnt = 0
for i=0 to n-1 do 
    for j=0 to i-1 do
        cnt <- cnt + 1
        let l1 = lines2.[i].[0] + " " + lines2.[j].[0]
        let l2 = lines2.[i].[2] + " " + lines2.[j].[2]
        let l3 = lines2.[i].[3] + " " + lines2.[j].[3]
        File.WriteAllLines(taskFolder + cnt.ToString() + ".txt", [| l1; l2; l3 |])
        //Directory.CreateDirectory(taskFolder + cnt.ToString()) |> ignore
        ()
    


//let ids = [1;2;3;];
let ids = Array.toList ("1 101 11 111 114 115 116 120 121 125 127 14 155 156 158 16 162 164 166 168 169 17 171 2 24 29 32 37 40 45 46 69 73 80 88 93 97 137 138 139 18 19 20 21 27 31 5 60 61 62".Split ' ') |> List.map int

let dir = @"D:\WORK\ImageData\task2\";
let taskid2str (id:int) = 
    let lines = File.ReadAllLines(dir + id.ToString() + ".txt");
    let line = lines.[1]
    line


taskid2str 10

ids |> List.iter (fun id ->
    printfn "%d %s" id (taskid2str id)
    )


//List.map (fun id -> (id, taskid2str id)) |> 




type Paper = {  
    id: int;   
    accept: bool;  
    country: string;    
    keywords: string array;    }


let accepts = Set([182;268;315;341;345;389;441;511;599;666;718;721;723;748;749;757;784;800;849;930;])



let papers = 
    let lines = File.ReadAllLines @"C:\Users\yin\Desktop\kdd-in.txt"
    lines
    |> Array.map (fun line ->
        let s = line.Split '\t'
        let id = int(s.[0])
        let country = s.[1]
        let keywords = 
            s.[2].Split([|';'|], System.StringSplitOptions.RemoveEmptyEntries)
            |> Array.map ( fun s -> s.Trim([|' '; '*'|]))
        { id = id;
          accept = accepts.Contains(id);
          country = country;
          keywords = keywords;
        }
        )



let getTitle (id:int) = 
    let ps = 
        let lines = File.ReadAllLines @"C:\Users\yin\Desktop\paperTitles.txt"
        lines
        |> Array.map (fun line ->
            let s = line.Split '\t'
            int (s.[0]), s.[1])
        |> Array.toList
    let rec find lst id = 
        match lst with
        | [] -> failwith "impossible"
        | x::xs -> if fst x = id then snd x else find xs id
    find ps id

let accs = papers |> Seq.filter (fun p -> p.accept)

let wordCntReducer (wseq: seq<_*int>) = 
    wseq |> Seq.groupBy (fun (id,cnt) -> id) |> Seq.map (fun (id, idseq) -> (id, idseq|>Seq.sumBy(fun (id,cnt) -> cnt)))
        (* test: wordCntReducer [1,1; 2,1; 1,1; 2,1; 2,2;] *)


seq { 
for p in accs do 
    let s = getTitle (p.id)
    yield! (s.Split ' ' |> Array.toSeq)
}
|> Seq.map (fun w -> (w,1))
|> wordCntReducer
|> Seq.sortBy (fun (w,c) -> (-c,w))
|> Seq.toArray

papers 
|> Seq.groupBy (fun p -> p.keywords.[0]) 
|> Seq.map (fun (c,s) -> c, Seq.length s, Seq.sumBy (fun p -> if p.accept then 1 else 0) s )
|> Seq.toList
|> List.sortBy (fun (c,cc,_) -> (-cc,c))
|> List.iter (fun (country, c1, c2) ->
    printfn "%s\t%d\t%d\t%f" country c1 c2 (100. * (float c2) / (float c1)))


let allKeywords = 
    [|
    for p in papers do 
        yield p.keywords.[0]
    |]
    


allKeywords 
|> Seq.map (fun w -> (w,1))
|> wordCntReducer
|> Seq.sortBy (fun (_, cnt) -> -cnt)
|> Seq.iter (fun (w, c) ->
    printfn "%s %d" w c
    )




